{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "151b7bd5",
   "metadata": {},
   "source": [
    "# 准备工作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b2898bdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import requests\n",
    "import requests_html\n",
    "from requests_html import HTMLSession\n",
    "from lxml import etree\n",
    "from lxml.html import fromstring\n",
    "from urllib.parse import urljoin\n",
    "from random import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import base64\n",
    "import json\n",
    "from PIL import Image\n",
    "\n",
    "import selenium\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "from selenium.webdriver.common.by import By"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "193a1384",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-46-b0d609aabe67>:20: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = webdriver.Chrome(executable_path=r'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver.exe', chrome_options=options)\n"
     ]
    }
   ],
   "source": [
    "#caps=dict()\n",
    "#caps[\"pageLoadStrategy\"] = \"none\"   # Do not wait for full page load\n",
    "\n",
    "opts = webdriver.ChromeOptions()\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "#opts.add_argument('blink-settings=imagesEnabled=false') #不加载图片, 提升速度\n",
    "#opts.add_argument('--headless') #浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败\n",
    "# opts.binary_location = \"C:\\portable\\PortableApps\\IronPortable\\App\\Iron\\chrome.exe\"\n",
    "# opts.binary_location = \"C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe\" #\"H:\\_coding_\\Gitee\\InternetNewMedia\\CapstonePrj2016\\chromedriver.exe\"  \n",
    "\n",
    "\n",
    "#指定文件下载路径\n",
    "options = webdriver.ChromeOptions()\n",
    "out_path = r'C:\\Users\\Ozl\\Desktop\\web_final\\download'  # 是你想指定的路径\n",
    "prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': out_path}\n",
    "options.add_experimental_option('prefs', prefs)\n",
    "driver = webdriver.Chrome(executable_path=r'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver.exe', chrome_options=options)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5350a0b6",
   "metadata": {},
   "source": [
    "# 打开知网"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "59eba5be",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://www.cnki.net/')\n",
    "time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cf4d324",
   "metadata": {},
   "source": [
    "# 检查知网登录情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c9f16398",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_id('Ecp_loginShowName1').get_attribute('innerHTML')\n",
    "time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f4e76f3",
   "metadata": {},
   "source": [
    "# 点击高级搜索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "aaa13e62",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('//*[@id=\"highSearch\"]').click()\n",
    "time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f628fb35",
   "metadata": {},
   "source": [
    "# 窗口信息检查"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "0f3e2ec7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CDwindow-6153C730D408B522DCB37B0584E2ABC8',\n",
       " 'CDwindow-39C43D17F20CE16B4C19AF639F1CA30B']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "driver.window_handles"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "461e96c1",
   "metadata": {},
   "source": [
    "# 窗口切换"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "517615f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-51-6c6d5ce6602d>:1: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "source": [
    "driver.switch_to_window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed7af875",
   "metadata": {},
   "source": [
    "# 点击学术期刊"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "47860452",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('/html/body/div[3]/div[1]/div/ul[1]/li[1]/a').click()\n",
    "time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f8358d2",
   "metadata": {},
   "source": [
    "# 点击专业检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "6f667cf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/ul/li[4]').click()\n",
    "time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbccafb7",
   "metadata": {},
   "source": [
    "# 点击勾选来源类别（CSSCI、CSCD）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "09e736a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#CSSCI\n",
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[5]/input').click()\n",
    "time.sleep(0.5)\n",
    "#CSCD\n",
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[6]/input').click()\n",
    "time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcde6b63",
   "metadata": {},
   "source": [
    "# 设置检索query并检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e92b55b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#设置感兴趣的query值\n",
    "query = 'SU = \"新媒体\" AND （TI = \"心理\" OR TI = \"心理学\"）'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "52644873",
   "metadata": {},
   "outputs": [],
   "source": [
    "#填进去\n",
    "element = driver.find_element_by_xpath('//textarea')\n",
    "element.clear()\n",
    "element.send_keys(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "ce41915e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#点击检索\n",
    "element = driver.find_element_by_xpath('//input[@value=\"检索\"]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97125a37",
   "metadata": {},
   "source": [
    "# 检查检索信息，更换页面文章数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "cb09539e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'共找到<em>4,042</em>条结果'"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#检索数量\n",
    "element = element = driver.find_element_by_xpath('//span[@class=\"pagerTitleCell\"]')\n",
    "element.get_attribute('innerHTML')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "49072dab",
   "metadata": {},
   "outputs": [],
   "source": [
    "#切换页面文章数量\n",
    "element = driver.find_element_by_xpath('//i[@class=\"icon icon-sort\"]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "5a6a89b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#点击\n",
    "element = driver.find_element_by_xpath('//div[@id=\"perPageDiv\"]//li[@data-val=\"50\"]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "936ff9e3",
   "metadata": {},
   "source": [
    "# 抓取页面信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "44929c98",
   "metadata": {},
   "outputs": [],
   "source": [
    "HOST = \"https://kns.cnki.net/\"\n",
    "html = driver.page_source\n",
    "soup = etree.HTML(html)\n",
    "tr_list= soup.xpath('//div[@id=\"gridTable\"]/table/tbody/tr')\n",
    "headers = {\n",
    "    \"Cookie\":'Ecp_ClientId=3200615112602316607; cnkiUserKey=03369524-5653-0508-82c6-2fdbcdb21fe4; RsPerPage=20; _pk_ref=%5B%22%22%2C%22%22%2C1607047617%2C%22https%3A%2F%2Fwww.cnki.net%2F%22%5D; Ecp_ClientIp=202.116.81.140; UM_distinctid=178689abc586d3-0e68ae30691535-5771133-1fa400-178689abc597f5; Ecp_loginuserjf=15014134753; Ecp_session=1; ASP.NET_SessionId=gt3afeursglwft12mavvuvzf; SID_kns8=123111; SID_kns_new=kns123106; CurrSortFieldType=desc; SID_kcms=124103; SID_kxreader_new=011121; Hm_lvt_6e967eb120601ea41b9d312166416aa6=1621474773; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22179876d894d472-0e2dc24144c1fc-2363163-2073600-179876d894e760%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%7D%2C%22%24device_id%22%3A%22179876d894d472-0e2dc24144c1fc-2363163-2073600-179876d894e760%22%7D; Hm_lvt_ba7af201fc75865e9846f701ccb53e6b=1621474773; SID_kns=025123113; SID_klogin=125144; Hm_lpvt_6e967eb120601ea41b9d312166416aa6=1621475961; Hm_lpvt_ba7af201fc75865e9846f701ccb53e6b=1621475961; Ecp_loginuserbk=GDZSDX; knsLeftGroupSelectItem=null5%3B9%3B; CurrSortField=%e8%a2%ab%e5%bc%95%2f(%e8%a2%ab%e5%bc%95%e9%a2%91%e6%ac%a1%2c%27integer%27); _pk_ref=%5B%22%22%2C%22%22%2C1622444666%2C%22https%3A%2F%2Fwww.cnki.net%2F%22%5D; _pk_ses=*; _pk_id=90adc29b-dc70-48ff-8dcc-60e442d5a980.1619271339.5.1622444668.1622444666.; Ecp_LoginStuts={\"IsAutoLogin\":false,\"UserName\":\"GZ0513\",\"ShowName\":\"%e4%b8%ad%e5%b1%b1%e5%a4%a7%e5%ad%a6%e5%8d%97%e6%96%b9%e5%ad%a6%e9%99%a2\",\"UserType\":\"bk\",\"BUserName\":\"\",\"BShowName\":\"\",\"BUserType\":\"\",\"r\":\"4SVIml\"}; LID=WEEvREcwSlJHSldSdmVqM1BLVW9SQVdSTUtBeko5WUtLcE8xUldvbWhxRT0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4IQMovwHtwkF4VYPoHbKxJw!!; c_m_LinID=LinID=WEEvREcwSlJHSldSdmVqM1BLVW9SQVdSTUtBeko5WUtLcE8xUldvbWhxRT0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4IQMovwHtwkF4VYPoHbKxJw!!&ot=05/31/2021 15:24:33; c_m_expire=2021-05-31 15:24:33',\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36'\n",
    "}\n",
    "def get_data(href):   #一个封装好的完善详细页链接的函数，从webmining大师JL那学来的\n",
    "    DbCode = re.findall(\"DbCode=(.*?)&\",href)[0]\n",
    "    dbname = re.findall(\"dbname=(.*?)&\",href)[0]\n",
    "    filename = re.findall(\"filename=(.*?)&\",href)[0]\n",
    "    href = f\"https://kns.cnki.net/kcms/detail/detail.aspx?dbcode={DbCode}&dbname={dbname}&filename={filename}\"\n",
    "    return href\n",
    "\n",
    "def download(href):   #一个封装好的获取PDF下载链接的函数，也是从webmining大师JL那学来的\n",
    "    page_source = requests.get(url=href,headers=headers).text\n",
    "    page_source_soup = etree.HTML(page_source)\n",
    "    download_url = page_source_soup.xpath('//li[@class=\"btn-dlpdf\"]/a/@href')\n",
    "    if download_url:\n",
    "        return urljoin(HOST,download_url[0])\n",
    "    else:\n",
    "        return \"无下载链接\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5c1aad1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_pages =18   #根据＞800篇的要求，每页50篇，设个18页应该OK了\n",
    "\n",
    "title_list = []\n",
    "author_list = []\n",
    "href_list = []\n",
    "download_url_list = []\n",
    "\n",
    "\n",
    "def main():\n",
    "    now_page_count = 1\n",
    "    while now_page_count < num_pages:\n",
    "        html = driver.page_source\n",
    "        soup = etree.HTML(html)\n",
    "        tr_list= soup.xpath('//div[@id=\"gridTable\"]/table/tbody/tr')\n",
    "        for tr in tr_list:\n",
    "            title = \"\".join(tr.xpath('td[@class=\"name\"]//text()')).strip().replace(\"\\n\",\"\").replace(\"                              网络首发\",\"\")\n",
    "            author = \",\".join(tr.xpath('td[@class=\"author\"]/a//text()')).strip().replace(\"\\n\",\"\")\n",
    "            href = tr.xpath('td[@class=\"name\"]/a/@href')[0]\n",
    "            href = get_data(href)\n",
    "            download_url = download(href)\n",
    "            title_list.append(title)\n",
    "            author_list.append(author)\n",
    "            href_list.append(href)\n",
    "            download_url_list.append(download_url)\n",
    "        now_page_count = now_page_count + 1\n",
    "        driver.find_element_by_xpath('//*[@id=\"PageNext\"]').click()\n",
    "        time.sleep(0.5)\n",
    "    data = {\"标题\":title_list,\"作者\":author_list,\"详细页链接\":href_list,\"PDF下载链接\":download_url_list}\n",
    "    data = pd.DataFrame(data)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4e82f8e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>作者</th>\n",
       "      <th>详细页链接</th>\n",
       "      <th>PDF下载链接</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>周作人阅读西方性心理学书籍考略——以蔼理斯为中心</td>\n",
       "      <td>黄艳芬</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>融合学习心理学的人类学习优化算法</td>\n",
       "      <td>孟晗,马良,刘勇</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>论中国早期电影心理学理论探索的三个阶段(1897—1949)</td>\n",
       "      <td>罗祎英</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>积极心理学在大学生心理健康教育中的运用——评《大学生心理健康教育：积极心理学的运用》</td>\n",
       "      <td>潘奕辰</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>论理论心理学的科学使命与历史命运</td>\n",
       "      <td>高申春,邢雷雷</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>845</th>\n",
       "      <td>科学主义的抵抑,人文主义的彰显——论描述心理学的基本特征</td>\n",
       "      <td>王申连,郭本禹</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846</th>\n",
       "      <td>基于心理学认知方式的陶瓷产品设计探讨</td>\n",
       "      <td>任远,张悦,胡强</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>847</th>\n",
       "      <td>社会建构论心理学:轮廓、流派和局限</td>\n",
       "      <td>郭慧玲</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>848</th>\n",
       "      <td>积极心理学对官兵挫折教育的启示</td>\n",
       "      <td>王玲莉,买跃霞</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>849</th>\n",
       "      <td>《中国临床心理学杂志》文献信息可视化分析</td>\n",
       "      <td>苏纯惠,张持晨,郑建中,胡伟红</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>850 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             标题               作者  \\\n",
       "0                      周作人阅读西方性心理学书籍考略——以蔼理斯为中心              黄艳芬   \n",
       "1                              融合学习心理学的人类学习优化算法         孟晗,马良,刘勇   \n",
       "2                论中国早期电影心理学理论探索的三个阶段(1897—1949)              罗祎英   \n",
       "3    积极心理学在大学生心理健康教育中的运用——评《大学生心理健康教育：积极心理学的运用》              潘奕辰   \n",
       "4                              论理论心理学的科学使命与历史命运          高申春,邢雷雷   \n",
       "..                                          ...              ...   \n",
       "845                科学主义的抵抑,人文主义的彰显——论描述心理学的基本特征          王申连,郭本禹   \n",
       "846                          基于心理学认知方式的陶瓷产品设计探讨         任远,张悦,胡强   \n",
       "847                           社会建构论心理学:轮廓、流派和局限              郭慧玲   \n",
       "848                             积极心理学对官兵挫折教育的启示          王玲莉,买跃霞   \n",
       "849                        《中国临床心理学杂志》文献信息可视化分析  苏纯惠,张持晨,郑建中,胡伟红   \n",
       "\n",
       "                                                 详细页链接  \\\n",
       "0    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "1    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "2    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "3    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "4    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "..                                                 ...   \n",
       "845  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "846  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "847  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "848  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "849  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "\n",
       "                                               PDF下载链接  \n",
       "0    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "1    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "2    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "3    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "4    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "..                                                 ...  \n",
       "845  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "846  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "847  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "848  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "849  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "\n",
       "[850 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#现在来输出数据表格\n",
    "data = main()\n",
    "display(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2ed0e73f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#让我们把这些数据导入excel，方便后续使用\n",
    "data.to_excel('cnki_webmining.xlsx',sheet_name=\"＞800篇\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4442b3f3",
   "metadata": {},
   "source": [
    "# PDF档下载及解决验证码(.pdf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93af876d",
   "metadata": {},
   "source": [
    "## 调用‘图鉴’API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1594eb73",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8sy8\n"
     ]
    }
   ],
   "source": [
    "#详情看接口文档 https://shimo.im/docs/MJm6EW3fdHknjjHX/read\n",
    "def base64_api(uname, pwd, img, typeid):\n",
    "    with open(img, 'rb') as f:\n",
    "        base64_data = base64.b64encode(f.read())\n",
    "        b64 = base64_data.decode()\n",
    "    data = {\"username\": uname, \"password\": pwd, \"typeid\": typeid, \"image\": b64}\n",
    "    result = json.loads(requests.post(\"http://api.ttshitu.com/predict\", json=data).text)\n",
    "    if result['success']:\n",
    "        return result[\"data\"][\"result\"]\n",
    "    else:\n",
    "        return result[\"message\"]\n",
    "    return \"\"\n",
    "\n",
    "#测试看看\n",
    "if __name__ == \"__main__\":\n",
    "    img_path = \"C:/Users/Ozl/Desktop/test.jpg\"\n",
    "    result = base64_api(uname='你的账号', pwd='你的密码', img=img_path, typeid=3)\n",
    "    print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91e6e037",
   "metadata": {},
   "source": [
    "## 调试验证码截图裁剪位置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "861ab651",
   "metadata": {},
   "outputs": [],
   "source": [
    "#由于直接获取验证码图片链接，图片是会变动的，所以采用截图的方法\n",
    "#截取全屏后，还需要定位截图里验证码的位置，并重新裁剪保存，好方便API识别出结果（这里的数值是根据我的电脑屏幕来调试的）\n",
    "\n",
    "# driver.save_screenshot('code.png')\n",
    "\n",
    "# img = driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "# print(img.location)#{'x': 684, 'y': 301}\n",
    "# print(img.size)#{'height': 36, 'width': 100}\n",
    " \n",
    "# left = img.location['x']+200\n",
    "# top = img.location['y']+140\n",
    " \n",
    "# right = img.location['x'] + img.size['width']+300\n",
    "# bottom = img.location['y'] + img.size['height']+160\n",
    " \n",
    "# photo = Image.open('code.png')\n",
    "# photo = photo.crop((left, top, right, bottom))\n",
    " \n",
    "# photo.save('code.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96d1d793",
   "metadata": {},
   "source": [
    "## 循环遍历链接下载pdf档（解决中间处理问题）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8cb8a3c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "#根据作业要求，我在开头启动driver的时候就指定了一个默认下载路径out_path = C:\\Users\\Ozl\\Desktop\\web_final\\download\n",
    "#这里我们用len（os.listdir(out_path)）的方法来确定下载的文件夹里有多少份文件，后续就可以用文件数量的变化来判断文件是否下载成功\n",
    "#预防出bug，现在文件夹里面已经有1份文档\n",
    "out_path =\"C:/Users/Ozl/Desktop/web_final/download\"\n",
    "for i in os.listdir(out_path):\n",
    "    exist = len(os.listdir(out_path))\n",
    "print(exist)\n",
    "#还需要设定一个下载后再数一次文件数量的值，用以判断\n",
    "# out_path =\"C:/Users/Ozl/Desktop/web_final/download\"\n",
    "# for i in os.listdir(out_path):\n",
    "#     new_exist=len(os.listdir(out_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "c35ad35a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-44-a621e8710a8a>:6: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = webdriver.Chrome(executable_path=r'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver.exe', chrome_options=options)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 1 篇下载成功\n",
      "第 2 篇下载成功\n",
      "第 3 篇下载成功\n",
      "第 4 篇下载成功\n",
      "第 5 篇下载成功\n",
      "第 6 篇下载成功\n",
      "第 7 篇下载成功\n",
      "第 8 篇下载成功\n",
      "第 9 篇下载成功\n",
      "第 10 篇下载成功\n",
      "第 11 篇下载成功\n",
      "第 12 篇下载成功\n",
      "第 13 篇下载成功\n"
     ]
    }
   ],
   "source": [
    "#用新打开的视窗来下载，依然自己设定默认下载路径\n",
    "options = webdriver.ChromeOptions()\n",
    "out_path = r'C:\\Users\\Ozl\\Desktop\\web_final\\download'  # 是你想指定的路径\n",
    "prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': out_path}\n",
    "options.add_experimental_option('prefs', prefs)\n",
    "driver = webdriver.Chrome(executable_path=r'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver.exe', chrome_options=options)\n",
    "#刚刚已经把pdf下载链接存到了excel表里所以直接用\n",
    "data = pd.read_excel(\"cnki_webmining.xlsx\")\n",
    "base = \"https://kns.cnki.net/\"\n",
    "download_url_list = data[\"PDF下载链接\"].to_list()\n",
    "for url in download_url_list[0:13]:\n",
    "    try:\n",
    "        driver.get(url)\n",
    "        time.sleep(6)\n",
    "        if driver.find_element_by_xpath('//*[@id=\"vImg\"]'):\n",
    "            driver.save_screenshot('code.png')\n",
    "            img = driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "            left = img.location['x']+200\n",
    "            top = img.location['y']+140\n",
    "            right = img.location['x'] + img.size['width']+300\n",
    "            bottom = img.location['y'] + img.size['height']+160\n",
    "            photo = Image.open('code.png')\n",
    "            photo = photo.crop((left, top, right, bottom))\n",
    "            photo.save('code.png')\n",
    "            img_path = \"C:/Users/Ozl/Desktop/web_final/code.png\"\n",
    "            result = base64_api(uname='你的账号',pwd='你的密码',img=img_path,typeid=3)\n",
    "            driver.find_element_by_id('vcode').send_keys(result) # 将验证码填入\n",
    "            time.sleep(0.5)\n",
    "            driver.find_element_by_xpath('/html/body/div/form/dl/dd/button').click() # 点击提交进行下载\n",
    "            #再设定一个下载后再数一次文件数量的值，用以判断\n",
    "            for i in os.listdir(out_path):\n",
    "                new_exist=len(os.listdir(out_path))\n",
    "#                 print(\"new_exist\")\n",
    "            print('第',download_url_list.index(url)+1,'篇下载成功')\n",
    "    except:\n",
    "            if new_exist == exist:\n",
    "                print('第',download_url_list.index(url)+1,'篇文件没有下载成功')\n",
    "            else:\n",
    "                print('第',download_url_list.index(url)+1,'篇下载成功')\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "727075f1",
   "metadata": {},
   "source": [
    "# 翻页，导出refworks文件（.txt）"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b4ed80f",
   "metadata": {},
   "source": [
    "## 前500篇"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "24fcd7ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "#回到第一页\n",
    "element = driver.find_element_by_xpath('//*[@id=\"total\"]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "334c0671",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#每页50篇文章，下载大于800篇至少需要翻17页，但知网规定一次最多选取500篇，所以这里设置（0，10）\n",
    "for i in range(0,10):\n",
    "    driver.find_element_by_xpath('//*[@id=\"selectCheckAll1\"]').click()\n",
    "    driver.find_element_by_id('PageNext').click()\n",
    "    time.sleep(5)\n",
    "    display(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "64b36d33",
   "metadata": {},
   "outputs": [],
   "source": [
    "#定位到要悬停的元素\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "move = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/a')\n",
    "#对定位到的元素执行悬停操作\n",
    "ActionChains(driver).move_to_element(move).perform()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "ab9b9dc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "move = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/a')\n",
    "ActionChains(driver).move_to_element(move).perform()\n",
    "driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]/a').click()#导出 refworks\n",
    "element=driver.find_element_by_xpath('//*[@id=\"litotxt\"]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "e95a68d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CDwindow-6153C730D408B522DCB37B0584E2ABC8',\n",
       " 'CDwindow-39C43D17F20CE16B4C19AF639F1CA30B',\n",
       " 'CDwindow-5FDF4F179C86567DD5ECFC4BF1136D44']"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#窗口信息检查\n",
    "driver.window_handles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "6a6de16b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-66-a8cb1d92e5f0>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[2])\n"
     ]
    }
   ],
   "source": [
    "#窗口切换\n",
    "driver.switch_to_window(driver.window_handles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "621718d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#导出 refworks\n",
    "element=driver.find_element_by_xpath('//*[@id=\"litotxt\"]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "875b7fbf",
   "metadata": {},
   "source": [
    "## 后300篇"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "a7a89a07",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-68-54452d0fe014>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[1])\n"
     ]
    }
   ],
   "source": [
    "#后300篇导出同理，先回到检索窗口\n",
    "driver.switch_to_window(driver.window_handles[1])\n",
    "#消除原先选定\n",
    "element=driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "7d379fa1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(0,6):\n",
    "    driver.find_element_by_xpath('//*[@id=\"selectCheckAll1\"]').click()\n",
    "    driver.find_element_by_id('PageNext').click()\n",
    "    time.sleep(5)\n",
    "    display(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "151cde03",
   "metadata": {},
   "outputs": [],
   "source": [
    "#定位到要悬停的元素\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "move = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/a')\n",
    "#对定位到的元素执行悬停操作\n",
    "ActionChains(driver).move_to_element(move).perform()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "ca196f38",
   "metadata": {},
   "outputs": [],
   "source": [
    "move = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/a')\n",
    "ActionChains(driver).move_to_element(move).perform()\n",
    "driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]/a').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "0bada713",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-75-a8cb1d92e5f0>:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  driver.switch_to_window(driver.window_handles[2])\n"
     ]
    }
   ],
   "source": [
    "#窗口切换\n",
    "driver.switch_to_window(driver.window_handles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "c1b844d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#导出 refworks\n",
    "element=driver.find_element_by_xpath('//*[@id=\"litotxt\"]/a')\n",
    "element.click()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
